#include "productDict.h"


#include <iostream>
#include <fstream>
#include <sstream>
#include <filesystem>

using std::cout;
using std::endl;
using std::cerr;
using std::ios;
using std::ofstream;
using std::ifstream;
using std::istringstream;
using std::stringstream;
namespace fs = std::filesystem;

std::u32string utf8ToUtf32(const std::string& utf8);
std::string utf32ToUtf8(char32_t c);


DictProducer::DictProducer(SplitTool* spliteTool, inicpp::IniManager* ini)
: _cuttor(spliteTool)
, _ini(ini)
{
    
}

void DictProducer::buildEnDict()
{
    
    //清洗英文数据，保存到新的文件中
    cleanoutEnglishData();
 
    //切割单词
    _cuttor->cutEnglishWord((*_ini)["rtsp"]["cleanedEnglishFile"], _words);
    //cutEnglishWord();

    //读取停用词文件
    //readStopWord("../data/stop_words_eng.txt");
    readStopWord((*_ini)["rtsp"]["stopEnglishWord"]);

    //导出去除停用词后的词频文件
    //outputWordFrequence("../data/en_word_frequent.txt");
    outputWordFrequence((*_ini)["rtsp"]["EnglishWordFreq"]);
    
    //创建英文词典位置索引
    createEnglishIndex("en");

}

//生成中文词频文件
void DictProducer::buildCnDict()
{
    vector<string> cnFiles;
    cnFiles = getFileNamesInDirectoryAndMove((*_ini)["rtsp"]["dirPath"]);
    //使用jieba分词，存放到_file中
    for(auto &tmpFilePath : cnFiles)
    {
        string fullPath = (*_ini)["rtsp"]["dirPath"] + tmpFilePath;
        _cuttor->cutChineseWord(fullPath, _words);
    }

    //读取停用词文件
    //readStopWord("../data/stop_words_zh.txt");
    readStopWord((*_ini)["rtsp"]["stopChineseWord"]);

    //导出去除停用词后的词频文件
    //outputWordFrequence("../data/cn_word_frequent.txt");
    outputWordFrequence((*_ini)["rtsp"]["ChineseWordFreq"]);

    createChineseIndex("cn");
}


//创建词典位置索引集合
void DictProducer::createEnglishIndex(string arg)
{   

    // 遍历词频向量中的每个元素
    for (int i = 0; i < _dict.size(); ++i) {
        string word = _dict[i].first;
        // 对于每个单词，遍历其中的每个字符
        for (char c : word) {
            // 将当前字符对应的向量下标插入到索引的集合中
            _index[string(1,c)].insert(i);
        }
    }
    
    string filename = "../data/" + arg + "_index_dict.txt";
    ofstream indexDict(filename);

     // 输出索引结果
    for (const auto& pair : _index) {
        indexDict << pair.first << " ";
        for (const int index : pair.second) {
            indexDict << index << " ";
        }
        indexDict << endl;
    }
}

void DictProducer::createChineseIndex(string arg) {
    // 遍历词频向量中的每个元素
    for (int i = 0; i < _dict.size(); ++i) {
        string word = _dict[i].first;
        // 对于每个单词，遍历其中的每个字符（对于中文，一个汉字算一个字符）
        for (char32_t c : utf8ToUtf32(word)) {
            string charStr = utf32ToUtf8(c);
            // 将当前字符对应的向量下标插入到索引的集合中
            _index[charStr].insert(i);
        }
    }

    string filename = "../data/" + arg + "_index_dict.txt";
    ofstream indexDict(filename);

    // 输出索引结果
    for (const auto& pair : _index) {
        indexDict << pair.first << " ";
        for (const int index : pair.second) {
            indexDict << index << " ";
        }
        indexDict << std::endl;
    }
}

//清洗英文数据，保存到新的文件中
void DictProducer::cleanoutEnglishData()
{
    ifstream enSourceIfs((*_ini)["rtsp"]["srcEnglishFile"]);//英文源文件
    ofstream enDsetOfs((*_ini)["rtsp"]["cleanedEnglishFile"]);

    string str_line;
    while(getline(enSourceIfs,str_line))
    {
        istringstream ss(str_line);
        string word;
        while(ss >> word)
        {
            string result;//用来写入的string
            for(char c : word)
            {
                if(isupper(c))
                {
                    result += tolower(c);//大写变小写
                }
                else if(ispunct(c))
                {
                    result += ' ';//符号变空格
                }
                else
                {
                    result += c;
                }
            }
            //将转换完的句子写入文件
            enDsetOfs << result << " ";
        }
        enDsetOfs << "\n";
    }

    enSourceIfs.close();
    enDsetOfs.close();
}



//读取停用词
void DictProducer::readStopWord(string filePath)
{
    ifstream stopWordIfs(filePath);
    //存放停用词

    char line[1024] = {0};
    while(stopWordIfs.getline(line,1024))
    {
        stringstream ss(line);
        string word;
        while(ss >> word)
        {
            _stop_words.insert(word);//将停用词插入
        }
    }

    stopWordIfs.close();
}

//导出词典位置索引
void DictProducer::outputWordFrequence(string filepath)
{

    map<string,int> tmpWordsFreq;//暂存词频

    //统计词频
    for(auto &word : _words)
    {
        //如果是停用词，跳过该词
        if(_stop_words.find(word.data()) != _stop_words.end())
        {
            continue;
        }
        else
        {
            if(tmpWordsFreq.find(word) == tmpWordsFreq.end())
            {
                tmpWordsFreq.insert({word,1});//字典中没有这个词,便插入这个词
            }
            else
            {
                tmpWordsFreq.find(word)->second++;//有就给词频+1
            }
        }      
    }

    //去除空格、\r和\n
    tmpWordsFreq.erase(" ");
    tmpWordsFreq.erase("\r");
    tmpWordsFreq.erase("\n");


    //将map转换为vector
    for(const auto &pair : tmpWordsFreq)
    {
        _dict.push_back(pair);
    }

    //将去除停用词后的词频统计写入文件
    ofstream enFreqOfs(filepath);
    for(auto &word : _dict)
    {
        //cout << word << "\n";
        enFreqOfs << word.first << " " << word.second << "\n";
    }

    enFreqOfs.close();
}



std::u32string utf8ToUtf32(const std::string& utf8) {
    std::u32string utf32;
    for (size_t i = 0; i < utf8.length();) {
        char32_t c;
        unsigned char firstByte = utf8[i];
        if ((firstByte & 0x80) == 0) {
            c = utf8[i++];
        } else if ((firstByte & 0xE0) == 0xC0) {
            c = ((utf8[i++] & 0x1F) << 6) | (utf8[i++] & 0x3F);
        } else if ((firstByte & 0xF0) == 0xE0) {
            c = ((utf8[i++] & 0x0F) << 12) | ((utf8[i++] & 0x3F) << 6) | (utf8[i++] & 0x3F);
        } else if ((firstByte & 0xF8) == 0xF0) {
            c = ((utf8[i++] & 0x07) << 18) | ((utf8[i++] & 0x3F) << 12) | ((utf8[i++] & 0x3F) << 6) | (utf8[i++] & 0x3F);
        } else {
            // 处理错误编码
            i++;
        }
        utf32 += c;
    }
    return utf32;
}

std::string utf32ToUtf8(char32_t c) {
    if (c <= 0x7F) {
        return std::string(1, static_cast<char>(c));
    } else if (c <= 0x7FF) {
        return std::string(1, static_cast<char>((c >> 6) | 0xC0)) + std::string(1, static_cast<char>((c & 0x3F) | 0x80));
    } else if (c <= 0xFFFF) {
        return std::string(1, static_cast<char>((c >> 12) | 0xE0)) + std::string(1, static_cast<char>(((c >> 6) & 0x3F) | 0x80)) + std::string(1, static_cast<char>((c & 0x3F) | 0x80));
    } else if (c <= 0x10FFFF) {
        return std::string(1, static_cast<char>((c >> 18) | 0xF0)) + std::string(1, static_cast<char>(((c >> 12) & 0x3F) | 0x80)) + std::string(1, static_cast<char>(((c >> 6) & 0x3F) | 0x80)) + std::string(1, static_cast<char>((c & 0x3F) | 0x80));
    } else {
        // 处理错误编码
        return "";
    }
}


//获取目录下所有文件名
vector<string> DictProducer::getFileNamesInDirectory(const string& directoryPath) {
    vector<string> fileNames;
    try {
        for (const auto& entry : fs::directory_iterator(directoryPath)) {
            if (fs::is_regular_file(entry)) {
                fileNames.push_back(entry.path().filename().string());
            }
        }
    } catch (const fs::filesystem_error& e) {
        cerr << "Error accessing directory: " << e.what() << "\n";
    }
    return fileNames;
}

// 修改后的函数，返回值使用移动构造
vector<string> DictProducer::getFileNamesInDirectoryAndMove(const string& directoryPath) {
    return vector<string>(getFileNamesInDirectory(directoryPath));
}



